package com.co.webmagic.task;

import com.co.webmagic.process.OldJobProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * @author ldl.plus
 * @date 2020年07月01日  9:47
 */
// @Component
public class OldDownJob {

    @Autowired
    private OldJobProcessor oldJobProcessor;

    /**
     * 设置拼接的url变量
     */
    private static int count = 841227222;

    private static final String urlPre = "https://www.bilibili.com/";

    @Scheduled(cron = "0/5 * * * * *")
    public void down() {
        Spider.create(oldJobProcessor)
                .addUrl(urlPre + "video/av" + count-- + "/")
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(1)
                .run();
        // Spider.create(new JobProcessor())
        //         .addUrl(urlPre + "ranking")
        //         .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
        //         .thread(1)
        //         .run();
    }
}
